#!/usr/bin/env python
# -*- coding: utf-8 -*-
import MySQLdb
from stemmer import Stemmer
import sys

reload(sys)
sys.setdefaultencoding("utf-8")

db = MySQLdb.connect(host="localhost", user="root", passwd="", db="wikiubb", charset='utf8')
cursor = db.cursor()

stemmerObj = Stemmer()

number_of_rows = 1000

iLimit = 50
offset = 0

it = 0

wordsIDMap = {}  
# Get words and id's
sql = "SELECT * FROM words"
cursor.execute(sql)
resultWordsTblData = cursor.fetchall()
for word in resultWordsTblData:
    wordsIDMap[word[1]] = word[0]  # key = word & value = word_id
    
#for k,v in wordsIDMap.items():
#    print k, v

while (offset < number_of_rows):
    sql = "SELECT * FROM text LIMIT %s OFFSET %s"   
    
    try :
        cursor.execute(sql, (iLimit, offset))
        results = cursor.fetchall()
        for doc in results:
            
            wordsMap = {}
            
            final_doc = stemmerObj.stemming(doc[1])
            for stemWord in final_doc:
                if stemWord not in wordsMap:
                    wordsMap[stemWord] = 1
                else:
                    wordsMap[stemWord] += 1
                    
            for k,v in wordsMap.items():
                unicodedK = unicode(k)
                if unicodedK in wordsIDMap:
                    #print k, v
                    print str(wordsIDMap[unicodedK]) + ' ' +  str(doc[0]) + ' ' + str(v)
                    sql = 'INSERT INTO indexedwords (wordID, documentumID, nr) VALUES (%s, %s, %s)' % (wordsIDMap[unicodedK], doc[0], v)
#                     print sql
                    cursor.execute(sql)
                    db.commit()
                else:
                    print k
                
            it = it + 1
            print it
    
        offset = offset + iLimit 
    except Exception as e:
        print "Error: unable to fetch data", e
 
cursor.close()
del cursor
db.close() 